﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Reinforcement;
using Neural;
using Probability;
using Computing;
namespace SAUlot
{
    public class AirplaneActorCritic: CCDecisionMaker
    {
        public event RaportForm.RaportThis RaportAppend;
        public event RaportForm.RaportTitle RaportSetTitle;
        public event RaportForm.RaportStep RaportSetStep;
        public event RaportForm.RaportClear RaportClearCharts;

        private ASampler Sampler;
        private MLPerceptron2 Actor; // w sensie Pi
        private MLPerceptron2 Critic; // w sensie V
        private double[] State; // 0:thata 1:theta_zadane 2:q 3:alpha
        public int Episode { get; set; }
        public int Step { get; set; }
        public double BetaFit {get; set;}
        public double BetaVt {get; set;}
        public double Gamma { get; set; }
        public double Deviation { get; set; }
        private double AdvisedAction;
        private Vector LastActorApprox;
        private int InputNumber;
        private int OutputNumber;
        private CellType ActorSecondLayerType;
        private CellType CriticSecondLayerType;
        private int FirstLayerNeuronNumber;
        private double AvgReward;

        public AirplaneActorCritic(int input, int output, double beta_ac, double beta_crit, double gamma, double std_dev, int fstlayer)
        {
            Sampler = new ASampler();
            LastActorApprox = new Vector(1);
            InputNumber = input;
            OutputNumber = output;
            BetaFit = beta_ac;
            BetaVt = beta_crit;
            Gamma = gamma;
            Deviation = std_dev;
            FirstLayerNeuronNumber = fstlayer;
            ActorSecondLayerType = Neural.CellType.Linear;
            CriticSecondLayerType = Neural.CellType.Linear;
            Step = 0;
            Episode = 0;
            //Init();
        }

        public AirplaneActorCritic(int input, int output, double beta_ac, double beta_crit, double gamma, double std_dev,int fstlayer, CellType actor, CellType critic)
        {
            Sampler = new ASampler();
            LastActorApprox = new Vector(1);
            InputNumber = input;
            OutputNumber = output;
            BetaFit = beta_ac;
            BetaVt = beta_crit;
            Gamma = gamma;
            Deviation = std_dev;
            FirstLayerNeuronNumber = fstlayer;
            ActorSecondLayerType = actor;
            CriticSecondLayerType = critic;
            Step = 0;
            Episode = 0;
            //Init();
        }

        public void Init()
        {
            Computing.Vector averages;
            Computing.Vector stddevs;

            RaportClearCharts();

            Actor = new Neural.MLPerceptron2();
            Actor.Build(InputNumber, new int[] { FirstLayerNeuronNumber, OutputNumber }, new CellType[] { Neural.CellType.Arcustangent, ActorSecondLayerType });
            averages = new Computing.Vector(InputNumber,0);
            stddevs = new Computing.Vector(InputNumber,1);
            Actor.SetInputDescription(averages, stddevs);
            Actor.InitWeights();

            Critic = new Neural.MLPerceptron2();
            Critic.Build(InputNumber, new int[] { 50, OutputNumber }, new CellType[] { Neural.CellType.Arcustangent,CriticSecondLayerType });
            averages = new Computing.Vector(InputNumber, 0);
            stddevs = new Computing.Vector(InputNumber, 1);
            Critic.SetInputDescription(averages, stddevs);
            Critic.InitWeights();
            Step = 0;
            Episode = 0;
        }

        public void CCStartInState(double[] state)
        {
            State = state.Take(InputNumber).ToArray<double>();
        }

        public void CCAdviseAction(ref double[] decision)
        {

            Actor.Approximate(new Vector(State), ref LastActorApprox);
            decision[0] = Sampler.SampleFromNormal(LastActorApprox[0], Deviation);
            AdvisedAction = decision[0];
        }

        public void CCThisHappened(double reward, double[] next_state)
        {
            Vector critic_approx = new Vector(1);
            Vector actor_approx  = new Vector(1);
            Critic.Approximate(new Vector(State), ref critic_approx);
            AvgReward += reward;
            Step += 1;
            double dt = 0.0;
            if (next_state[next_state.Length - 1] == 1)
            {
                AvgReward = AvgReward / Step;
                Episode += 1;
                RaportAppend(2, AvgReward);
                AvgReward = 0;
                Step = 0;
                dt = reward - critic_approx[0];
            }
            else
            {
                Vector critic_approx_next = new Vector(1);
                Critic.Approximate(new Vector(next_state.Take(InputNumber).ToArray<double>()), ref critic_approx_next);
                dt = reward + Gamma * critic_approx_next[0] - critic_approx[0];
            }

            RaportSetTitle(0, "dt");
            RaportSetStep(0, 0.5);
            RaportAppend(0, dt);

            RaportSetTitle(1, "Reward");
            RaportSetStep(1, 0.5);
            RaportAppend(1, reward);

            RaportSetTitle(2, "Average Reward");
            RaportSetStep(2, 0.005);
            //RaportAppend(2, LastActorApprox[0]);

            RaportSetTitle(3, "AdvisedAction");
            RaportSetStep(3, 0.5);
            RaportAppend(3, AdvisedAction);
            //poprawka aktora
            double[] dec = new double[1];

            dec[0] = ((AdvisedAction-LastActorApprox[0])/(Math.Pow(Deviation,2)));
            Vector action = new Vector(dec);
            Vector actor_weights = new Vector(Actor.WeightsCount);
            Actor.BackPropagateGradient(action, ref actor_weights);
            Actor.AddToWeights(actor_weights, BetaFit * dt);
            //poprawka krytyka
            Vector critic_weights = new Vector(Critic.WeightsCount);
            Critic.BackPropagateGradient(new Vector(1,dt), ref critic_weights);
            Critic.AddToWeights(critic_weights, BetaVt * dt);

            State = next_state.Take(InputNumber).ToArray<double>();
            //BetaFit = BetaFit * 0.9;
            //BetaVt = BetaVt * 0.9;
        }
    }
}
